In [2]:
cd ..
In [3]:
import pylearn2.utils
import pylearn2.config
import theano
import neukrill_net.utils
import numpy as np
%matplotlib inline
import matplotlib.pyplot as plt
import holoviews as hl
%load_ext holoviews.ipython
import sklearn.metrics
import pickle
import neukrill_net.utils
import neukrill_net.encoding as enc
import neukrill_net.taxonomy as t
In [6]:
%env THEANO_FLAGS = device=gpu0,floatX=float32,base_compiledir=~/.theano/stonesoup0
In [4]:
settings = neukrill_net.utils.Settings("settings.json")
run_settings = neukrill_net.utils.load_run_settings('run_settings/experiment_setting_colnorms_higher_aug.json',
settings, force=True)
model = pylearn2.utils.serial.load(run_settings['pickle abspath'])
In [7]:
# format the YAML
yaml_string = neukrill_net.utils.format_yaml(run_settings, settings)
# load proxied objects
proxied = pylearn2.config.yaml_parse.load(yaml_string, instantiate=False)
# pull out proxied dataset
proxdata = proxied.keywords['dataset']
# force loading of dataset and switch to test dataset
proxdata.keywords['force'] = True
proxdata.keywords['training_set_mode'] = 'test'
proxdata.keywords['verbose'] = False
# then instantiate the dataset
dataset = pylearn2.config.yaml_parse._instantiate(proxdata)
In [8]:
if hasattr(dataset.X, 'shape'):
N_examples = dataset.X.shape[0]
else:
N_examples = len(dataset.X)
batch_size = 500
while N_examples%batch_size != 0:
batch_size += 1
n_batches = int(N_examples/batch_size)
#n_classes = len(settings.classes)
In [9]:
model.set_batch_size(batch_size)
X = model.get_input_space().make_batch_theano()
Y = model.fprop(X)
if type(X) == tuple:
f = theano.function(X,Y)
else:
f = theano.function([X],Y)
In [10]:
augment = 1
y = np.zeros((N_examples*augment,188))
# get the data specs from the cost function using the model
pcost = proxied.keywords['algorithm'].keywords['cost']
cost = pylearn2.config.yaml_parse._instantiate(pcost)
data_specs = cost.get_data_specs(model)
i = 0
In [11]:
for _ in range(augment):
# make sequential iterator
iterator = dataset.iterator(batch_size=batch_size,num_batches=n_batches,
mode='even_sequential', data_specs=data_specs)
for batch in iterator:
if type(X) == tuple:
y[i*batch_size:(i+1)*batch_size,:] = f(batch[0],batch[1])
else:
y[i*batch_size:(i+1)*batch_size,:] = f(batch[0])
i += 1
In [12]:
af = run_settings.get("augmentation_factor",1)
if af > 1:
y_collapsed = np.zeros((int(N_examples/af), 188))
for i,(low,high) in enumerate(zip(range(0,dataset.y.shape[0],af),
range(af,dataset.y.shape[0]+af,af))):
y_collapsed[i,:] = np.mean(y[low:high,:], axis=0)
y = y_collapsed
# and collapse labels
labels = dataset.y[range(0,dataset.y.shape[0],af)]
elif augment > 1:
y_collapsed = np.zeros((N_examples,188))
# different kind of augmentation, has to be collapsed differently
for row in range(N_examples):
y_collapsed[row,:] = np.mean(np.vstack([r for r in
y[[i for i in range(row,N_examples*augment,N_examples)],:]]),
axis=0)
y = y_collapsed
labels = dataset.y
else:
labels = dataset.y
In [196]:
y.shape
Out[196]:
In [115]:
predictions = np.zeros(y.shape)
np.copyto(predictions, y)
predictions.shape
Out[115]:
Predict with the classes.
In [29]:
class_predictions = np.zeros((y.shape[0], 121))
np.copyto(class_predictions, predictions[:, :121])
labels = labels[:, :121]
logloss = sklearn.metrics.log_loss(labels,class_predictions)
print("Log loss: {0}".format(logloss))
Now let's try with superclasses.
In [117]:
superclasses = np.zeros((y.shape[0], 38))
np.copyto(superclasses, predictions[:,121:(121+38)])
superclasses.shape
Out[117]:
In [31]:
import neukrill_net.taxonomy as t
only_leaf_children = []
hier = enc.get_hierarchy(settings)
layer = t.TaxonomyLayer(1)
for s in hier[1]:
flag = True
for key, values in t.superclasses.items():
for i, v in enumerate(values):
if v == s and i != 1:
flag = False
if flag:
if s not in only_leaf_children:
only_leaf_children.append(s)
In [32]:
only_leaf_children
Out[32]:
In [33]:
import neukrill_net.taxonomy as t
layer = t.TaxonomyLayer(1)
backmap = {}
for i, c in enumerate(settings.classes):
j = int(np.where(np.array(hier[1]) == layer[c])[0])
if hier[1][j] in only_leaf_children:
try:
backmap[j] += [i]
except:
backmap[j] = [i]
In [34]:
backmap
Out[34]:
In [35]:
nr = np.zeros(class_predictions.shape)
np.copyto(nr, class_predictions)
weight = 0.01
for index,r in enumerate(superclasses):
if index%1000 == 0:
print index
for i,j in enumerate(r):
priors = []
if i in backmap:
d = sum([settings.class_priors[k] for k in backmap[i]])
for a in backmap[i]:
priors.append(settings.class_priors[a]/d)
nr[index,backmap[i]] = nr[index, backmap[i]]*(1-weight) + (weight)*j*np.array(priors)
In [36]:
nr.shape
Out[36]:
With weight=0 it's the same.
In [18]:
logloss = sklearn.metrics.log_loss(labels,nr)
print("Log loss: {0}".format(logloss))
With weight=0.01.
In [37]:
logloss = sklearn.metrics.log_loss(labels,nr)
print("Log loss: {0}".format(logloss))
In [38]:
superclass2_children = {}
settings = neukrill_net.utils.Settings("settings.json")
hier = enc.get_hierarchy(settings)
for s in hier[2]:
children = []
for key, values in t.superclasses.items():
for i, v in enumerate(values):
if s == v:
if i > 0:
if values[i-1] not in children:
children.append(values[i-1])
superclass2_children[s] = children
In [41]:
hier[1]
Out[41]:
In [39]:
superclass2_children
Out[39]:
In [ ]:
layer = t.TaxonomyLayer(1)
weight = 0.01
for index, row in enumerate(nr):
for key, value in superclass2_children.items():
proportion = []
leaf = []
for i,v in enumerate(value):
if v in only_leaf_children:
j = int(np.where(np.array(hier[1]) == v)[0])
proportion.append(superclasses[index, j])
else:
if v in settings.classes:
leaf.append(i)
for l in leaf:
denom = sum(proportion)
proportion = proportion/denom
col = int(np.where(np.array(hier[2]) == key)[0])
row[i] = (superclasses2[row, col] - proportion)
if index%1000 == 0:
print index
Log loss plot
In [37]:
N = y.shape[0]
In [47]:
logloss = lambda x: -(1./N)*np.log(x[0][x[1]])
In [48]:
ilabels = np.where(labels)[1]
In [49]:
ll = []
for i,(p,l) in enumerate(zip(y,ilabels)):
ll.append((i,logloss((p,l))))
In [52]:
h=plt.hist(zip(*ll)[1], bins=50)
In [56]:
worst = int(np.where(np.array(ll)[:,1]==max(np.array(ll)[:,1]))[0])
worst
Out[56]:
In [59]:
ll[1512]
Out[59]:
What does the worst log loss correspond to?
In [58]:
settings.classes[ilabels[worst]]
Out[58]:
In [64]:
misclass = int(np.where(y[worst,:121] == max(y[worst,:121]))[0])
misclass
Out[64]:
It was misclassified as:
In [65]:
settings.classes[misclass]
Out[65]:
Looks like this:
In [60]:
hl.Image(dataset.X[worst])
Out[60]:
In [70]:
for d in np.where(ilabels == misclass)[0][:10]:
try:
c += hl.Image(dataset.X[d])
except:
c = hl.Image(dataset.X[d])
c
Out[70]:
In [71]:
hier[2]
Out[71]:
In [74]:
np.copyto(predictions, y)
y.shape
Out[74]:
In [109]:
%env THEANO_FLAGS = 'device=gpu3,floatX=float32,base_compiledir=~/.theano/stonesoup3'
In [111]:
labels = labels[:, :121]
logloss = sklearn.metrics.log_loss(labels,predictions[:, :121])
print("Log loss: {0}".format(logloss))
In [116]:
superclasses2 = predictions[:,(121+38):(121+38+16)]
superclasses2.shape
Out[116]:
Get children of each second parent superclass.
In [42]:
superclass2_children = {}
settings = neukrill_net.utils.Settings("settings.json")
hier = enc.get_hierarchy(settings)
for s in hier[2]:
children = []
for key, values in t.superclasses.items():
for i, v in enumerate(values):
if s == v:
if i > 0:
if values[i-1] not in children:
children.append(values[i-1])
superclass2_children[s] = children
In [43]:
superclass2_children
Out[43]:
In [72]:
nr = class_predictions
weight = 0.01
nr.shape
Out[72]:
In [148]:
layer = t.TaxonomyLayer(1)
row = nr[1,:]
weight = 0.01
for key, value in superclass2_children.items():
for i,v in enumerate(value):
if v in only_leaf_children:
update = {}
for i, c in enumerate(settings.classes):
if v == layer[c]:
print v
update[i] = settings.class_priors[i]
denom = sum(v for k,v in update.items())
indUpdate = [k for k,v in update.items()]
print row[indUpdate]
row[indUpdate] = row[indUpdate] * (1-weight) + [v for k,v in update.items()/denom * weight]
print row[indUpdate]
In [157]:
row == predictions[1,:121]
Out[157]:
In [62]:
only_leaf_children
Out[62]:
Matt's way of updating predictions:
In [178]:
layer = t.TaxonomyLayer(1)
hier = enc.get_hierarchy(settings)
priors = np.zeros(len(hier[1]))
superclass_children = [[] for k in range(len(hier[1]))]
# For each class
for i, c in enumerate(settings.classes):
# Find index of its first parent in the 1-of-k encoding array
j = int(np.where(np.array(hier[1]) == layer[c])[0])
# Add class's prior for that superclass group
priors[j] += settings.class_priors[i]
# Record index of this class as a child or superclass
superclass_children[j].append(i)
In [179]:
priors
Out[179]:
In [180]:
superclass_children
Out[180]:
In [182]:
new_class_predictions = np.zeros(class_predictions.shape)
new_class_predictions.shape
Out[182]:
In [183]:
new_class_predictions = np.zeros(class_predictions.shape)
for index, row in enumerate(class_predictions):
for i, c in enumerate(settings.classes):
j = int(np.where(np.array(hier[1]) == layer[c])[0])
new_class_predictions[index, i] = superclasses[index, j] * (settings.class_priors[i] / priors[j])
In [184]:
new_class_predictions.shape
Out[184]:
In [185]:
updated_predictions = np.zeros(class_predictions.shape)
updated_predictions = predictions[:, :121] * (1-weight) + weight * new_class_predictions
In [186]:
labels = labels[:, :121]
logloss = sklearn.metrics.log_loss(labels, predictions[:,:121])
print("Log loss: {0}".format(logloss))
In [187]:
logloss = sklearn.metrics.log_loss(labels, updated_predictions)
print("Log loss: {0}".format(logloss))
In [188]:
weight
Out[188]:
In [189]:
results = []
weights = np.logspace(-5,-1, 20)
for weight in weights:
updated_predictions = np.zeros(121)
updated_predictions = predictions[:, :121] * (1-weight) + weight * new_class_predictions
logloss = sklearn.metrics.log_loss(labels, updated_predictions)
results.append(logloss)
In [190]:
results
Out[190]:
In [191]:
plt.semilogx(weights, results)
Out[191]:
In [195]:
predictions = np.zeros(y.shape)
np.copyto(predictions, y)
Out[195]:
In [159]:
class_predictions.shape
Out[159]:
Make a big list with predictions for each vector.
In [170]:
all_predictions = [[] for k in range(len(hier))]
start = 0
for i in range(len(hier)):
end = len(hier[i])
all_predictions[i] = predictions[:, start:(start + end)]
start = start + end
Check the dimensions are right.
In [173]:
[y.shape for y in all_predictions]
Out[173]:
Make a list for priors for all vectors.
In [152]:
priors = [[] for k in range(len(hier))]
for i in range(len(hier)):
priors[i] = np.zeros(len(hier[i]))
priors[0] = settings.class_priors
Set it with class priors in the zeroth vector.
In [153]:
priors
Out[153]:
Propagate up to compute superclass group priors in all vectors.
In [154]:
# For each class
for index, h in enumerate(hier):
if (index < 5):
layer = t.TaxonomyLayer(index + 1)
for i, c in enumerate(settings.classes):
# Find index of its parent in 1-of-k encodings
j = int(np.where(np.array(hier[index + 1]) == layer[c])[0])
# Add class's prior for that superclass group
priors[index + 1][j] += settings.class_priors[i]
In [155]:
priors
Out[155]:
Check they sum to 1.
In [156]:
[sum(p) for p in priors]
Out[156]:
In [166]:
new_predictions = [np.zeros(class_predictions.shape) for k in range(len(hier))]
new_predictions[0] = class_predictions
Out[166]:
In [177]:
new_predictions[0][0,0]
Out[177]:
In [168]:
# For each array of new predictions
for pred in range(len(new_predictions)):
# No need to update the zeroth one
if pred > 0:
layer = t.TaxonomyLayer(pred)
# For each image
for index, row in enumerate(new_predictions[pred]):
# For each class
for i, c in enumerate(settings.classes):
# Find the parent
j = int(np.where(np.array(hier[pred]) == layer[c])[0])
new_predictions[pred][] = superclasses[index, j] * (settings.class_priors[i] / priors[j])